{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "zaksum eval.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/theamrzaki/text_summurization_abstractive_methods/blob/master/Implementation%20B%20(Pointer%20Generator%20seq2seq%20network)/zaksum_eval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "HY3btgJWEqBP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Google Drive"
      ]
    },
    {
      "metadata": {
        "id": "tdcKjPVpEjk8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "14ffae67-4fc4-4dfe-cd75-96505c64e06e"
      },
      "cell_type": "code",
      "source": [
        "#https://stackoverflow.com/questions/47744131/colaboratory-can-i-access-to-my-google-drive-folder-and-file\n",
        "\n",
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()\n",
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}\n",
        "\n",
        "!mkdir -p drive\n",
        "!google-drive-ocamlfuse drive"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "E: Package 'python-software-properties' has no installation candidate\n",
            "Selecting previously unselected package google-drive-ocamlfuse.\n",
            "(Reading database ... 110845 files and directories currently installed.)\n",
            "Preparing to unpack .../google-drive-ocamlfuse_0.7.1-0ubuntu3~ubuntu18.04.1_amd64.deb ...\n",
            "Unpacking google-drive-ocamlfuse (0.7.1-0ubuntu3~ubuntu18.04.1) ...\n",
            "Setting up google-drive-ocamlfuse (0.7.1-0ubuntu3~ubuntu18.04.1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "uZlHlsJAE75n",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Parser"
      ]
    },
    {
      "metadata": {
        "id": "GBUTmucLFHy4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "article=[]\n",
        "reference=[]\n",
        "summary =[]\n",
        "\n",
        "import xml.etree.ElementTree\n",
        "e = xml.etree.ElementTree.parse(\"drive/Colab Notebooks/Model 4 generator/result_pointer_4_1_2019_9_39pm.xml\").getroot()\n",
        "\n",
        "for atype in e.findall('example/article'):\n",
        "    article.append(atype.text)\n",
        "    \n",
        "for atype in e.findall('example/reference'):\n",
        "     reference.append(atype.text)\n",
        "    \n",
        "for atype in e.findall('example/summary'):\n",
        "     summary.append(atype.text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XJinxjb1Gwxg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### zaksum"
      ]
    },
    {
      "metadata": {
        "id": "a23wJfc6Gzg4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440
        },
        "outputId": "8f1ea9b5-f8ac-40b7-d297-7ff0e3632dd3"
      },
      "cell_type": "code",
      "source": [
        "!pip install sumeval\n",
        "!python -m spacy download en"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sumeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/a4/cd9da40dd7f32a2141808ba3af7101f0fcb0abfed918defd2df05b1410ac/sumeval-0.1.7.tar.gz (44kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 2.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: plac>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from sumeval) (0.9.6)\n",
            "Collecting sacrebleu>=1.1.7 (from sumeval)\n",
            "  Downloading https://files.pythonhosted.org/packages/37/51/bffea2b666d59d77be0413d35220022040a1f308c39009e5b023bc4eb8ab/sacrebleu-1.2.12.tar.gz\n",
            "Collecting typing (from sacrebleu>=1.1.7->sumeval)\n",
            "  Downloading https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl\n",
            "Building wheels for collected packages: sumeval, sacrebleu\n",
            "  Running setup.py bdist_wheel for sumeval ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/f1/7c/18/177331e7a5154401ab5fec8f3c6d2062508c38d832aaa3d5e2\n",
            "  Running setup.py bdist_wheel for sacrebleu ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/ea/0a/7d/ddcbdcd15a04b72de1b3f78e7e754aab415aff81c423376385\n",
            "Successfully built sumeval sacrebleu\n",
            "Installing collected packages: typing, sacrebleu, sumeval\n",
            "Successfully installed sacrebleu-1.2.12 sumeval-0.1.7 typing-3.6.6\n",
            "Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n",
            "\n",
            "\u001b[93m    Linking successful\u001b[0m\n",
            "    /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n",
            "    /usr/local/lib/python3.6/dist-packages/spacy/data/en\n",
            "\n",
            "    You can now load the model via spacy.load('en')\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7gN0BNbaG35g",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#https://github.com/chakki-works/sumeval\n",
        "#https://github.com/Tian312/awesome-text-summarization\n",
        "\n",
        "from sumeval.metrics.rouge import RougeCalculator\n",
        "from sumeval.metrics.bleu import BLEUCalculator\n",
        "\n",
        "def eval_rouges(refrence_summary,model_summary):\n",
        "    rouge = RougeCalculator(stopwords=True, lang=\"en\")\n",
        "\n",
        "    rouge_1 = rouge.rouge_n(\n",
        "                summary=model_summary,\n",
        "                references=refrence_summary,\n",
        "                n=1)\n",
        "\n",
        "    rouge_2 = rouge.rouge_n(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary],\n",
        "                n=2)\n",
        "    \n",
        "    rouge_l = rouge.rouge_l(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary])\n",
        "    \n",
        "    # You need spaCy to calculate ROUGE-BE\n",
        "    \n",
        "    rouge_be = rouge.rouge_be(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary])\n",
        "\n",
        "    bleu = BLEUCalculator()\n",
        "    bleu_score = bleu.bleu( summary=model_summary,\n",
        "                        references=[refrence_summary])\n",
        "    \n",
        "    return rouge_1, rouge_2,rouge_l,rouge_be,bleu_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "i6atapqsG52A",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#https://pymotw.com/2/xml/etree/ElementTree/create.html\n",
        "\n",
        "bleu_arr = []\n",
        "rouge_1_arr  = []\n",
        "rouge_2_arr  = []\n",
        "rouge_L_arr  = []\n",
        "rouge_be_arr = []\n",
        "\n",
        "from xml.etree import ElementTree\n",
        "from xml.dom import minidom\n",
        "from functools import reduce\n",
        "\n",
        "def prettify(elem):\n",
        "    \"\"\"Return a pretty-printed XML string for the Element.\n",
        "    \"\"\"\n",
        "    rough_string = ElementTree.tostring(elem, 'utf-8')\n",
        "    reparsed = minidom.parseString(rough_string)\n",
        "    return reparsed.toprettyxml(indent=\"  \")\n",
        "  \n",
        "from xml.etree.ElementTree import Element, SubElement, Comment\n",
        "\n",
        "top = Element('ZakSum')\n",
        "\n",
        "def zaksum(article,reference,summary_array,default_path):\n",
        "  comment = Comment('Generated by Amr Zaki')\n",
        "  top.append(comment)\n",
        "\n",
        "  i=0\n",
        "  for summ in summary_array:\n",
        "    example = SubElement(top, 'example')\n",
        "    article_element   = SubElement(example, 'article')\n",
        "    article_element.text = article[i]\n",
        "\n",
        "    reference_element = SubElement(example, 'reference')\n",
        "    reference_element.text = reference[i]\n",
        "\n",
        "    summary_element   = SubElement(example, 'summary')\n",
        "    summary_element.text = summ\n",
        "\n",
        "    rouge_1, rouge_2,rouge_L,rouge_be,bleu_score = eval_rouges(reference[i],summ )\n",
        "\n",
        "    eval_element = SubElement(example, 'eval')\n",
        "    bleu_score_element = SubElement(eval_element,'BLEU', {'score':str(bleu_score)})\n",
        "    ROUGE_1_element  = SubElement(eval_element, 'ROUGE_1' , {'score':str(rouge_1)})\n",
        "    ROUGE_2_element  = SubElement(eval_element, 'ROUGE_2' , {'score':str(rouge_2)})\n",
        "    ROUGE_L_element  = SubElement(eval_element, 'ROUGE_l' , {'score':str(rouge_L)})\n",
        "    ROUGE_be_element  = SubElement(eval_element,'ROUGE_be', {'score':str(rouge_be)})\n",
        "\n",
        "    bleu_arr.append(bleu_score) \n",
        "    rouge_1_arr.append(rouge_1) \n",
        "    rouge_2_arr.append(rouge_2) \n",
        "    rouge_L_arr.append(rouge_L) \n",
        "    rouge_be_arr.append(rouge_be) \n",
        "\n",
        "    i+=1\n",
        "\n",
        "  top.set('bleu', str(reduce(lambda x, y: x + y,  bleu_arr) / len(bleu_arr)))\n",
        "  top.set('rouge_1', str(reduce(lambda x, y: x + y,  rouge_1_arr) / len(rouge_1_arr)))\n",
        "  top.set('rouge_2', str(reduce(lambda x, y: x + y,  rouge_2_arr) / len(rouge_2_arr)))\n",
        "  top.set('rouge_L', str(reduce(lambda x, y: x + y,  rouge_L_arr) / len(rouge_L_arr)))\n",
        "  top.set('rouge_be', str(reduce(lambda x, y: x + y, rouge_be_arr) / len(rouge_be_arr)))\n",
        "\n",
        "  with open(default_path + \"result.xml\", \"w+\") as f:\n",
        "    print(prettify(top), file=f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "O7cG7_wZHdOf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Run"
      ]
    },
    {
      "metadata": {
        "id": "ZCMS9Bb_HQH4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "zaksum(article,reference,summary,\"drive/Colab Notebooks/Model 4 generator/\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}